{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Noisy DQN\n",
    "\n",
    "**Paper**: [Noisy Networks for Exploration](https://arxiv.org/pdf/1706.10295.pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import gym\n",
    "\n",
    "\n",
    "from src.utils.Config import Config\n",
    "from src.utils.Logging import Logger\n",
    "from src.components.memory import ReplayBuffer, PrioritiesedReplayBuffer\n",
    "from src.networks.models import QNetwork, NoisyQNetwork, NoisyDualingQNetwork\n",
    "from src.utils.misc import train, watch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: \t\tEnvSpec(CartPole-v1)\n",
      "win condition: \t195.0\n",
      "state space\t(4,)\n",
      "action space\t2\n",
      "device: \tcpu\n",
      "seed: \t\t123456789\n",
      "n_episodes: \t2000\n",
      "max_t: \t\t1000\n",
      "eps_start: \t1.0\n",
      "eps_end: \t0.01\n",
      "eps_decay: \t0.995\n",
      "eps_greedy: \tTrue\n",
      "noisy: \t\tFalse\n",
      "tau: \t\t0.001\n",
      "gamma: \t\t0.99\n",
      "lr: \t\t0.0005\n",
      "memory: \t<class 'src.components.memory.PrioritiesedReplayBuffer'>\n",
      "batch_size: \t64\n",
      "buffer_size: \t100000\n",
      "lr_annealing: \tFalse\n",
      "learn_every: \t4\n",
      "double_dqn: \tFalse\n",
      "model: \t\t<class 'src.networks.models.NoisyQNetwork'>\n",
      "save_loc: \tNone\n",
      "Logging at: logs/CartPole-v1/experiment-2020-04-19_15_31_48\n"
     ]
    }
   ],
   "source": [
    "config = Config()\n",
    "\n",
    "config.env = gym.make('CartPole-v1')\n",
    "\n",
    "config.win_condition = 195.0\n",
    "config.memory = PrioritiesedReplayBuffer\n",
    "config.model = NoisyQNetwork\n",
    "config.print_config()\n",
    "config.noisy = True\n",
    "\n",
    "logger = Logger(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epi: 100\t Frame: 2103\tAverage Score: 21.0300\tMean: 47.0000\tDuration: 0.2511\t#t_s: 46.0000\n",
      "Epi: 200\t Frame: 4328\tAverage Score: 22.2500\tMean: 74.0000\tDuration: 0.6075\t#t_s: 73.0000\n",
      "Epi: 300\t Frame: 6350\tAverage Score: 20.2200\tMean: 22.0000\tDuration: 0.1155\t#t_s: 21.0000\n",
      "Epi: 400\t Frame: 8747\tAverage Score: 23.9700\tMean: 18.0000\tDuration: 0.0873\t#t_s: 17.0000\n",
      "Epi: 500\t Frame: 11362\tAverage Score: 26.1500\tMean: 82.0000\tDuration: 0.4036\t#t_s: 81.0000\n",
      "Epi: 600\t Frame: 23722\tAverage Score: 123.6000\tMean: 191.0000\tDuration: 1.2750\t#t_s: 190.0000\n",
      "Epi: 645\t Frame: 33073 \tAverage: 196.0900\tMean: 250.0000\tDuration: 1.3075\t#t_s: 249.0000\n",
      "Environment Solved in 184.6342 seconds !\n"
     ]
    }
   ],
   "source": [
    "train(config, logger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "score_replay_buffer = logger.score\n",
    "average_score_replay_buffer = logger.average_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# watch(config, logger.log_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: \t\tEnvSpec(CartPole-v1)\n",
      "win condition: \t195.0\n",
      "state space\t(4,)\n",
      "action space\t2\n",
      "device: \tcpu\n",
      "seed: \t\t123456789\n",
      "n_episodes: \t2000\n",
      "max_t: \t\t1000\n",
      "eps_start: \t1.0\n",
      "eps_end: \t0.01\n",
      "eps_decay: \t0.995\n",
      "eps_greedy: \tTrue\n",
      "noisy: \t\tFalse\n",
      "tau: \t\t0.001\n",
      "gamma: \t\t0.99\n",
      "lr: \t\t0.0005\n",
      "memory: \t<class 'src.components.memory.ReplayBuffer'>\n",
      "batch_size: \t64\n",
      "buffer_size: \t100000\n",
      "lr_annealing: \tFalse\n",
      "learn_every: \t4\n",
      "double_dqn: \tFalse\n",
      "model: \t\t<class 'src.networks.models.NoisyQNetwork'>\n",
      "save_loc: \tNone\n",
      "Logging at: logs/CartPole-v1/experiment-2020-04-19_15_34_54\n"
     ]
    }
   ],
   "source": [
    "config = Config()\n",
    "\n",
    "config.env = gym.make('CartPole-v1')\n",
    "\n",
    "config.win_condition = 195.0\n",
    "config.memory = ReplayBuffer\n",
    "config.model = NoisyQNetwork\n",
    "config.print_config()\n",
    "config.noisy = True\n",
    "\n",
    "logger = Logger(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epi: 100\t Frame: 1302\tAverage Score: 13.0200\tMean: 10.0000\tDuration: 0.0186\t#t_s: 9.0000\n",
      "Epi: 200\t Frame: 2877\tAverage Score: 15.7500\tMean: 10.0000\tDuration: 0.0364\t#t_s: 9.0000\n",
      "Epi: 300\t Frame: 4390\tAverage Score: 15.1300\tMean: 19.0000\tDuration: 0.0444\t#t_s: 18.0000\n",
      "Epi: 400\t Frame: 5880\tAverage Score: 14.9000\tMean: 10.0000\tDuration: 0.0266\t#t_s: 9.0000\n",
      "Epi: 500\t Frame: 7243\tAverage Score: 13.6300\tMean: 9.0000\tDuration: 0.0189\t#t_s: 8.0000\n",
      "Epi: 600\t Frame: 8733\tAverage Score: 14.9000\tMean: 9.0000\tDuration: 0.0183\t#t_s: 8.0000\n",
      "Epi: 700\t Frame: 10414\tAverage Score: 16.8100\tMean: 43.0000\tDuration: 0.1847\t#t_s: 42.0000\n",
      "Epi: 800\t Frame: 12477\tAverage Score: 20.6300\tMean: 18.0000\tDuration: 0.0433\t#t_s: 17.0000\n",
      "Epi: 900\t Frame: 18347\tAverage Score: 58.7000\tMean: 108.0000\tDuration: 0.2393\t#t_s: 107.0000\n",
      "Epi: 1000\t Frame: 36869\tAverage Score: 185.2200\tMean: 255.0000\tDuration: 0.5994\t#t_s: 254.0000\n",
      "Epi: 1014\t Frame: 39792 \tAverage: 195.3200\tMean: 197.0000\tDuration: 0.4793\t#t_s: 196.0000\n",
      "Environment Solved in 100.9852 seconds !\n"
     ]
    }
   ],
   "source": [
    "train(config, logger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "score_replay_per_buffer = logger.score\n",
    "average_score_replay_per_buffer = logger.average_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl0HFeV+PHvVau1W5u1WJZsy4vseI2dOBsJSYjJOkxCgAkJWwgZDPwCExgGBpg5wMxv4AfDEoYzEAgQEpgQkpAQQjBkQsgCJHFix/smS7ZlS9bS2tVae3m/P17Jlm3Z1l5Vrfs5p091V1V33eqSbr1+9eo9McaglFIqcSW5HYBSSqnJpYleKaUSnCZ6pZRKcJrolVIqwWmiV0qpBKeJXimlEpwmeqWUSnCa6JVSKsFpoldKqQSX7HYAAAUFBaa8vNztMJRSylc2b97cbIwpPNt6nkj05eXlbNq0ye0wlFLKV0SkZiTrnbXqRkTmiMjzIrJbRHaJyN3O/HwReVZE9jvTPGe+iMh3RaRKRLaLyHnj2xWllFLjMZI6+ijwaWPMMuBi4C4RWQZ8DnjOGFMBPOe8BrgeqHAe64F7JzxqpZRSI3bWRG+MqTfGvOE87wL2AKXATcCDzmoPAm93nt8E/MxYrwK5IlIy4ZErpZQakVG1uhGRcmANsBEoNsbUO4sagGLneSlwZMjbap15SimlXDDiRC8iWcDjwCeNMZ1Dlxnbqf2oOrYXkfUisklENoVCodG8VSml1CiMKNGLSBCb5B8yxjzhzG4crJJxpk3O/DpgzpC3lznzTmCMuc8Ys9YYs7aw8Kytg5RSSo3RSFrdCPATYI8x5ttDFj0F3O48vx34zZD5H3Ba31wMdAyp4lFKKTXFRtKO/lLg/cAOEdnqzPsC8DXgURG5E6gBbnGWbQBuAKqAHuCOCY1YKXWCxzfXEonFufXCuW6HojzqrIneGPMXQE6zeN0w6xvgrnHGpZQaoZ+/WkNqcpImej96+lOQNx8u/YdJ3Yz2daOUj8Xihr0NnSyfneN2KGq0Brphy0PQNfk125rolfKxA6EwfZE4y2dnux2KGq1X74VYP5zztknflCZ6pXzsiS22QduaubkuR6JGxRj463eh7AKY96ZJ35wmeqV87NUDLaydl8eCwiy3Q1GjseV/oL8DFl8LcrpLoBNHE71SPhWLG/bWd7GiVOvnfWfbw5AyAy78yJRsThO9Uj51sDlMbySmid5velrhyGuw9g5Im5prK5rolfKpXUdtTyR6IdZnKp+BeARWvHPKNqmJXimf2nqkndTkJBYVaf28r1T+ATKLYNaqKdukJnqlfOqlyhAXzs8nGNB/Y9/Y9STsfhLOuQGSpu646V+IUj7U1NlHdaibyyu0Q0Bf2fxTyCuH674+pZvVRK+UD+1r7AJgeanWz/tGPAZHXoeKayCYNqWb1kSvlA9VNYUBtH7eTxq2Q6Tb3iQ1xTTRK+VD1aEw2WnJFGaluh2KGon+LrjvSvtcE71SaiT21ndRUTwDmYK7KtUEeO2+48/zyqd885rolfKZSCzOjroOVs/R/m18Y+8GSM2BO34/JV0enEwTvVI+U9PSTX9Ue6z0jZ5WOPoGXPyxKenAbDia6JXymcpGeyF2cfEMlyNRI3LwRTBxWHiVayFoolfKZyobuxCBhdpjpffF47Y74tQcKD3ftTBGMjj4/SLSJCI7h8x7RES2Oo9Dg2PJiki5iPQOWfaDyQxeqeloT30n8/IzSE8JuB2KOpu9v7XVNguvhMBIhuieHCPZ8gPAfwM/G5xhjHn34HMR+RbQMWT9amPM6okKUCl1nDGGzTVtekesX2x7xE7/5tuuhjGSwcFfEpHy4ZaJbdt1C+Be5ZNS00hNSw/N4QHOL89zOxR1NpE+2Pc7+zyzwNVQxltH/2ag0Rizf8i8+SKyRUReFJE3j/PzlVJDbKppA2DtvHyXI1FndeRVO52CMWHPZryVRrcBDw95XQ/MNca0iMj5wJMistwY03nyG0VkPbAeYO7cueMMQ6npYWddB5kpASq06wPv2/hDSM+Hd/zI7UjGXqIXkWTgHcAjg/OMMf3GmBbn+WagGlg83PuNMfcZY9YaY9YWFmp9o1Ijsb+pi0VFWSQl6R2xnmYMHHwJVrwDUjLcjmZcVTdvBfYaY2oHZ4hIoYgEnOcLgArgwPhCVEoNqmoKs1BL897XcQQGwlC83O1IgJE1r3wYeAVYIiK1InKns+hWTqy2Abgc2O40t/wV8FFjTOtEBqzUdNXZF6Gxs5+KIr1RyvOa9tpp4VJ343CMpNXNbaeZ/8Fh5j0OPD7+sJRSJ9OuiX2kabedFp3jbhwOvTNWKZ/QRO8job0wowTSvdEMVhO9Uj5R3RQmJTmJOXnpboeizqZpDxR6ozQPmuiV8o39TWEWFGSSrIOBe1s8Ds2VUOSN+nnQRK+Ub2iLG59or4FIjyZ6pdTo9EViHGnr0Rul/KBpj516pMUNaKJXyheqQ2GM0QuxvnDwRUhO80wbetBEr5QvaIsbn4jHYNeTsHCdJ+6IHaSJXikfqG4KkyQwvyDT7VDUmRzZCOEGWPkutyM5gSZ6pXxgf1OYeTMzSU3WwUY8rX6bnZZf5m4cJ9FEr5QPVDWFdehAP2jYCZmFkFXkdiQn0ESvlMdFYnEOtXRTUayJ3tNiUdtjZfEKtyM5hSZ6pTyupqWHSMywSEv03nboJeg4DMtucjuSU2iiV8rjqpq6AG1x43mv/gAkSRO9Umr09jdq00pfOPoGLLkBMrw3zKMmeqU8bn9TmNLcdDJTxzvyp5o0zVXQHYJ5l7odybA00SvlcZWNXXoh1uuqnrXTZTe6G8dpaKJXysPiccPB5m69EOt1TbshYyZkl7odybBGMpTg/SLSJCI7h8z7sojUichW53HDkGWfF5EqEdknItdOVuBKTQd17b30R+Paa6XXHd0Cs1aCeHPQ9pGU6B8Arhtm/j3GmNXOYwOAiCzDjiW73HnP9wcHC1dKjV51yF6I1ZulPKw/DI27oOxCtyM5rbMmemPMS8BIB/i+CfilMabfGHMQqAK8u/dKeVx1qBuAhYXax41n1W8FE4eytW5HclrjqaP/uIhsd6p2BgdGLAWODFmn1pmnlBqD6lCY3Iwg+ZkpboeiTie01049eEfsoLEm+nuBhcBqoB741mg/QETWi8gmEdkUCoXGGIZSia2mpZvymZmIR+t+FRCqhJQsyJ7tdiSnNaZEb4xpNMbEjDFx4Eccr56pA+YMWbXMmTfcZ9xnjFlrjFlbWFg4ljCUSngt4QEKZ6S6HYY6k+ZKKKjw7IVYGGOiF5GSIS9vBgZb5DwF3CoiqSIyH6gAXhtfiEpNX63dA+RnaLWNp7VUwcwKt6M4o7PeaiciDwNXAgUiUgt8CbhSRFYDBjgEfATAGLNLRB4FdgNR4C5jTGxyQlcqsRljaOsZID9LE71nRfuhoxZmLnQ7kjM6a6I3xtw2zOyfnGH9rwBfGU9QSik42tFHJGaYnZPmdijqdNpqAAN55W5HckZ6Z6xSHrWrrgOA5aU5LkeiTqvtoJ3mzXc3jrPQRK+UR+082kmSwNJZ2W6Hok6n9YCd5muiV0qNwa66DhYVZZGeojeXe1b9Njt0YKa3Ww5qolfKo3Ye7WDFbK228bTaTVC61tNNK0ETvVKe1NY9QGNnP0tLtNrGs3rboGU/lJ3vdiRnpYleKQ+qbLTDBy6eNcPlSNRp1W2207IL3I1jBDTRK+VBg4l+SbEmes+q3QwIzD7P7UjOShO9Uh60r7GL7LRkirO1+wPPOroFChZDmver1zTRK+VBlQ1hlsyaoZ2ZeVnjLpjl3R4rh9JEr5THGGPY19jFYq228a6+Tug4DEXL3I5kRDTRK+UxTV39dPRGWKIXYr2raY+dFi93N44R0kSvlMfsa3Ba3GiJ3ruadtmpJnql1FhoixsfaNwNqdmQM+fs63qAJnqlPGZvQxdFM1LJ0+EDvatxFxQt9fwdsYM00SvlMZWNXVo/72XG2Kobn1yIBU30SnlKPG6o1BY33tZ5FPo6fFM/D5rolfKUI2099EXiWj/vZU277VRL9EqpsTjW4karbryrcbDFTQIlehG5X0SaRGTnkHnfEJG9IrJdRH4tIrnO/HIR6RWRrc7jB5MZvFKJZrDFTUVRlsuRqNNq2g3ZpZCe53YkIzaSEv0DwHUnzXsWWGGMWQVUAp8fsqzaGLPaeXx0YsJUanrY1xhmTn46malnHc5ZucEYOPwKzFrldiSjctZEb4x5CWg9ad7/GmOizstXgbJJiE2paaeyoUvr572sbjO0H4alf+t2JKMyEXX0HwJ+P+T1fBHZIiIvisibT/cmEVkvIptEZFMoFJqAMJTyt4FonOpQWFvceNnOJyCQAkvf5nYkozKuRC8i/wJEgYecWfXAXGPMGuAfgV+IyLB9eBpj7jPGrDXGrC0s9PZ4i0pNhYPN3UTjRtvQe1mdM3Rgmr+GeBxzoheRDwJvA95rjDEAxph+Y0yL83wzUA0snoA4lUp4rx+yNaSrynJdjkSdVvN+KPRfShtToheR64DPAjcaY3qGzC8UkYDzfAFQARyYiECVSnS7jnaQn5lC+cwMt0NRw+lugd5WO9iIz5z10r6IPAxcCRSISC3wJWwrm1TgWWdghFedFjaXA/8uIhEgDnzUGNM67AcrpU5QHepmYWGmDjbiVc2Vdjqzwt04xuCsid4Yc9sws39ymnUfBx4fb1BKTUcHQmGuOqfI7TDU6dRvtdNZK92NYwz0zlilPKC9Z4Dm8ACL9EYp72rcCZlFkF3idiSjpoleKQ+oagoDaKL3srYayF/gdhRjooleKQ84lugLtWmlZ7Udgvz5bkcxJprolfKAqqYwqclJlOalux2KGk50ADpqIa/c7UjGRBO9Uh5QFQqzoDCLQJK2uPGk9hrAQO48tyMZE030SnlAVVNY6+e9rH6bnfposJGhNNEr5bK+SIy69l4WFma6HYo6ndpNEMzw1WAjQ2miV8plR1p7MAbmF2ii96y6TVCyGgL+7D5aE71SLqtpsb2IzM3Xrg88KToA9duh7Hy3IxkzTfRKuaym1Sb6eTO1RO9JjTsg1g+lmuiVUmN0uKWbrNRk8jKCboeihlP3hp2WrnU3jnHQRK+Uy2pae5ibn6GdmXlV7SbIKoYc/w6kp4leKZfVtPTohVgvGxxsxMcnYk30SrkoGotT29bDPO2D3pt626ClCkrPczuScdFEr5SL6jv6iMQM5Xoh1pvqNttpmX/r50ETvVKuOtTSDaAleq+qewMQmK0leqXUGB1y2tCXax29N9VugsIlkJbtdiTjMqJELyL3i0iTiOwcMi9fRJ4Vkf3ONM+ZLyLyXRGpEpHtIuLvU6FSk6imuZu0YBJFM1LdDkWdzBjnQqx/288PGmmJ/gHgupPmfQ54zhhTATznvAa4HjsoeAWwHrh3/GEqlZgOtfRQPlPHifWk1gPQ0wJzLnQ7knEbUaI3xrwEnDzI903Ag87zB4G3D5n/M2O9CuSKiP/G3lJqChxq6dauD7zqyGt2WjZNEv1pFBtj6p3nDUCx87wUODJkvVpnnlJqiIFonEPN3VQUa/fEntSwA5LTbR29z03IxVhjjAHMaN4jIutFZJOIbAqFQhMRhlK+crC5m2jcsLhYhw/0pJb9MHMRJAXcjmTcxpPoGwerZJxpkzO/DpgzZL0yZ94JjDH3GWPWGmPWFhYWjiMMpfypsrELgIoiTfSe1FwJBYvcjmJCjCfRPwXc7jy/HfjNkPkfcFrfXAx0DKniUUo59jd2kSSwQAcc8Z5IH7QfhoLFbkcyIUbUi76IPAxcCRSISC3wJeBrwKMicidQA9zirL4BuAGoAnqAOyY4ZqUSQmVjmHkzM0kL+r9qIOG0HgATh5kVbkcyIUaU6I0xt51m0bph1jXAXeMJSqnpoLKpiwodJ9abWvbbaUFiJHq9M1YpF/RHY9S09OiFWK/qarBTH3dNPJQmeqVccCDUTSxutGmlV/U4tw2l5bobxwTRRK+UCwZb3GiJ3qN6WyEtx7eDgZ9ME71SLthR20FqchKLtI7em3paIT3f7SgmjCZ6pVywva6DZbOzCQb0X9CTupsgM3Hu79G/MqWmmDGG3Uc7WVma43Yo6nQ66iAncXpu0USv1BRr7Own3B/VppVeFY9BZx1ka6JXSo1RdSgMwMJCTfSedHQLRPtg9hq3I5kwmuiVmmIHBhO9lui9afujkJwGC69yO5IJo4leqSlW2RgmKzVZR5Xyquo/wfzLIUNb3SilxmjXUdviRkeV8qCOWtv9wYIr3Y5kQmmiV2oKxeKG3fWdrJitLW486cCLdjr/CnfjmGCa6JWaQgdCYfoicVaUZrsdihrOgRds+/miZW5HMqE00Ss1hXYe7QBguZbovanmZZh3KSQlVmpMrL1RyuN21nWSmpzEQh1sxHu6GqCzFub4fzDwk2miV2oK7azrYGlJNsna9YH31L5up6Vr3Y1jEuhfm1JTJB63XR9o/bxH7XkaUmbA7NVuRzLhNNErNUWOdvTS1R/lnFma6D2ntx12PAbn3grJiXd/w5g7WxaRJcAjQ2YtAL4I5AIfBkLO/C8YYzaMOUKlEsSBUDeAdk3sRQdfAhODFe9wO5JJMeZEb4zZB6wGEJEAUAf8GjsY+D3GmG9OSIRKJYjBPm400XvQob9AMBPKLnA7kkkxUVU364BqY0zNBH2eUglnf1OYnPQgMzNT3A5Fnay12g4EHgi6HcmkmKhEfyvw8JDXHxeR7SJyv4jkDfcGEVkvIptEZFMoFBpuFaUSyov7Qpw/L0+7PvCilmrIm+d2FJNm3IleRFKAG4HHnFn3Agux1Tr1wLeGe58x5j5jzFpjzNrCwrGN5NLRG+GFfU20hPvH9H6lpkpb9wB17b1csmCm26Gok7UehLaDMOcityOZNBNRor8eeMMY0whgjGk0xsSMMXHgR8Ck3X1wsLmbD/70dbbVtk/WJpSaEAeaB7sm1hulPKdus53Ov9zdOCbRRCT62xhSbSMiJUOW3QzsnIBtDCvg/ASOxydrC0pNjOom2+JGBxvxoPqtEEiBwnPcjmTSjLnVDYCIZAJXAx8ZMvs/RWQ1YIBDJy2bUINVnTFjJmsTSk2IfY1dpCQnUZaX4XYo6mSHN0Lx8oS9EAvjTPTGmG5g5knz3j+uiEYhkCSD25yqTSo1JhsPtnDe3Nxjf7PKI9qPQO1r8JZ/dTuSSeXrO2OTnCJ9TKtulId19ETYdbSTSxYUuB2KOllor52WX+ZuHJPM14l+sF+ouJbolYdVhbowBlaVadfEnlPzMkgAipa6Hcmk8nWiH2yPrIleednh1h4A5uSnuxyJOoExsOcpmP9mSM91O5pJ5etEH9BEr3xgb0MXwYDohVivadoDLVWw7Ca3I5l0vk70Wkev/GDr4XZWlOaQFgy4HYoa6qAzPmzFNe7GMQX8nei1jl75QFNXP6W5Wm3jOUc2Qs4cyClzO5JJ5+9Ef+yGKU30yruau/opyEq8Ps5978hrCTls4HB8negH2yRrnlde1ReJ0dUfpXCGJnpP6W6GzjqYvcbtSKaErxO93hmrvG6wD/p5M/VCrKfUb7PTWavcjWOK+DrRD7a60TtjlVfta+gCYEnxDJcjUSc4+oadzlrpbhxTxNeJ/nirG030ypu2HmknMyXAAu3MzFv2boCS1ZCR73YkU8LfiV7r6JXHbTvSzsqyHO3jxkvaamyJfvnb3Y5kyvg70Tv/O9rqRnlRPG7Y09DFylLt+sBTdv/GTpdpoveF461uNNEr72kO9zMQjTM3Xy/EesrBl6BwKeTPdzuSKePrRH+sjl4TvfKg2vZeAEpy9GYpT+k8Oq2SPCRIoteqG+VFgy1uKor1QqyndB2FGSVnXy+B+DzR26nmeeVFO+o6mJGWrFU3XtLfBb1tkFPqdiRTalwjTAGIyCGgC4gBUWPMWhHJBx4ByrHDCd5ijGkb77ZONlhHr80rlRftqutgxeycY91pKw+o3WSnBUvcjWOKTVSJ/i3GmNXGmLXO688BzxljKoDnnNcTTvSGKeVRkVjctrjRwUa85fUfQzBj2vRxM2iyqm5uAh50nj8ITFo7pkCS6MVY5Tk1LT0MROOcM0vviPWM/i6o+iOc9wHIKnI7mik1EYneAP8rIptFZL0zr9gYU+88bwCKJ2A7w0oSraNX3tPaPQCgnZl5ySvfh2gfrLzF7Uim3Ljr6IHLjDF1IlIEPCsie4cuNMYYETklFTsnhfUAc+fOHfPGk0S01Y3ynLYem+jzMlJcjkQds+V/YNHVUHa+25FMuXGX6I0xdc60Cfg1cCHQKCIlAM60aZj33WeMWWuMWVtYWDjm7QeSRG+YUp7T7iT63Iygy5EoAMIh6DgMC65wOxJXjCvRi0imiMwYfA5cA+wEngJud1a7HfjNeLZzJkkiOpSg8pzGzn4A8jO1RO8JB16w09LpV5qH8VfdFAO/dlq/JAO/MMb8QUReBx4VkTuBGmDSKsVsHb2W6JW3bK/tYEFhJhkpE1E7qsZt2y8gLXfaDDRysnH9FRpjDgDnDjO/BVg3ns8eqSStulEedLi1m4oivSPWE+JxqP4TLL8ZgtOzOwpf3xkLdvARTfTKa+o7+rSPG6/Y8Zidpue5G4eL/J/ok4RoTBO98o6WcD9dfVFKctLcDkUB7Ntgp1d+wd04XOT7RB8MJDEQ1auxyjv+vL8ZgEsXFbgciaJuM+z9HZz/Qcgae+s+v/N9ok9NTqJfm90oD9nb0EUwIHpXrNsOvAA/ugoCQbj8s25H4yrfJ/qUZC3RK+/oj8b4wYvVVBTNIDng+38vf3vxG3Z6/denXW+VJ/P9X6ImeuUlv9/RAMDli6dvNYEnDHTDkY1w6d22b5tpzv+JXuvolUc0dfbxyUe2kiTw2WunVze4nrP1FxCPwPzpeSfsyXyf6FODSQxoHb3ygN9ut/34ffqaJSQlaR/0rql5BTb8k03ymuiBBEj0WqJXXrH1SDuluenc9ZZFbocyvf3qDju94RsQ0DuTYWJ6r3SV1tErtzV09PHTvx7kmZ0N3LByltvhTG+7n4Kueii7EAq1+mxQAiT6gFbdKFfd/cstbDzYCsDtbyp3N5jpbO8GeOrj9vk7fuhuLB6jVTdKjcNvttYdS/K5GUHWzJ2+t9m7auvD8MvbIDkdPvYy5C9wOyJP8X2JPis1QGdfxO0w1DR0qLmbf358O2V56fzsQxfqaFJuiMfgtfvg+a/CjBL4+z9CTpnbUXmO7xN9QVYqXX1R+qMxUpMDboejpomegSjv/fFGgklJPPqRS5idqx2YueKF/wcvfQNmrYR3/kST/Gn4vuqmwClFtYQHXI5ETRcdPRFu+9FG6tp7+c6tqzXJu+XQX+Clb8Kqd8NH/6IXX8/A9yX6WU4PgQebu/UfTk2atu4BHtt8hNcOtvHHPY0A3L2ugnVLJ23ce3UmPa3wyPshIx+u+5rb0Xie7xP9RfPzCQaEP+9v1t4C1aTY39jFu37wCh29ERYVZXFBeR6XVxTyiXUVboc2fT37Rehrhw//ySZ7dUa+T/QZKcksmTWDnXUdboeiElAsbvjYQ28QDAhPf+IyVpTmuB2S2vIQbPk5XPrJaTs04GiNuY5eROaIyPMisltEdonI3c78L4tInYhsdR43TFy4w1tYmMXh1p7J3oyaZr73fBULv7CBqqYwn79+qSZ5t0X64OlPwYbPQNkFsO6LbkfkG+Mp0UeBTxtj3hCRGcBmEXnWWXaPMeab4w9vZGblpNHQ0YcxBmegcqXGrLGzj9vvf429DV0A3PWWhfztubNdjmqaO7oVHn0/tB+GktVw439DkrayG6kxJ3pjTD1Q7zzvEpE9gCudPpdkpzEQixPq6qcoW4dv8xJjDC3dA+SmBz3dP3ssbnipMsTu+k5+/OcDtPVE+My1S1h/+QKCHo57WjjwAjz8Hoh0w9vugbUfcjsi35mQOnoRKQfWABuBS4GPi8gHgE3YUn/bMO9ZD6wHmDt37ri2v2y2/Um9vbaDty7TRO8Fe+o7+f4L1WzYUU8sbsjLCLKqLJdrl8+iODuVjJRkfvBiNe29ERYUZJISSCIvM4XymRnsru9kIBpnIBpn2exs1szN5fx5E3/BLdTVzwv7mjjU0s2vNtfS2NkPwKKiLL51y7lcdY62qHFVzSvwsxshNmDveL3rdShc7HZUviTGjG9gbRHJAl4EvmKMeUJEioFmwAD/FygxxpzxFLx27VqzadOmMcfQOxBj5Zef4SNXLOAz154z5s+ZKMYYegZixI0hKzV5xNVJ/dEYD288zLlzckd0K70xhv5onJRAkie6xQ33R0kJJLHpUCvv+fFGwA7e/qm3VrCzrpNtte3Ud/Sd8J6lJdmEuvpo64kQi5/+b/Gvn7uK0hE0n43G4rx2qJXDLT28cbiNtp4IxtjvKtwfZWlJNndeNp+CrFTWfesFjnb0kSRw8YKZXLG4kHVLi5hfkEXAA9/ntBXth1f+G577d/v6nLfB1f8OMxe6G5cHichmY8zas603rhK9iASBx4GHjDFPABhjGocs/xHw9Hi2MRLpKQGWlmTzRk37ZG/qjDp6I1SHwnzmsW1Uh7oBKMhKYWFhFndcOp/LFxfQ3hPhvpcOsK22nYFonMsqCkgPBmjrHuDJrUfp6LXdORTOSOWDbyonPRggGBBq23pJDgh9EVtFtbehk9buAZrDAxRnp3LPu1ezdl4+KcmTV80QicXp6Y+RkxFk25F2NuyoJxIzNHb2sfVIO3XtvQQDQiRmE/bJVR+RWJyXq1s4GApT39HH+y+ZR1lexrHPrw6FOdrey7KSbNJTAjR3DbCpppV/fHQbl37tT9y8ppQrlxRy6aICalq66YvEGTyH1rf3Udfey/1/PUh7z/EuMRYUZiJAWjBAWjDAAy8f4oGXDx2L86s3r+SGlbPIzUiZtO9NjVBXAyDw+8/C7ieh8BxbVTPvTW5H5ntjLtGLLaY+CLQaYz45ZH6JU3+PiHwKuMgYc+uZPmu8JXqAL/5mJ7/aXMuWL149oV0hxOPmlNJyXyRGXXsvCwuzaAn3s7ehi5TkJP758e0cCHWTFkxH5Hy1AAAOy0lEQVTiE1dVEInF2VPfyfN7Q6f0sHluWQ69kRiVjeFj8xYUZPLuC+awYUc922pP31w0SSBuYFZ2Gjetmc0vNh6mqy/KoqIsfv1/3kRmSjJ90RgZKckYY+jojZCTHjzll0VNSzf/8bs9GGO4e91iVpbZKrCuvgjtPRFmpCUTSBLSggHaeyLc8cBr7KzrZPnsbPY1dCECqckBMlMDnDMrm3PLcgj3x5idm8a1y2cxJz9juPBH7XvPV/GNZ/aRkx48diI8kwUFmdS29fLIRy4+5ZfR73fU8x+/28Ps3DTed/E8blo9vccSdU1Hne2+4LzbbfcFf/wybLz3+PLV74Mbv6sXXM9ipCX68ST6y4A/AzuAwSz2BeA2YDW26uYQ8JHBxH86E5HoX6wMcfv9r/H9957HDStLhl2nozdCuD/Kk1vq2HK4jSuXFPGmhTNJDQYI90UpyEohPzMFEaG9Z4DPP7GDP+xqoCDLdrPQOxAjMzVwrC737atn80Jl6IQS5D9evZirlxWztCT7hO1urmnlr1UtRGJxrlsxizctPH5zVzxuiBlDcpIcS8bGGLoHYkRjcXYf7aQkNx0B8jJSyE4/sTqotq2HZ3c38h+/20MgSYjFDbG4YUZqMl39UQCKs1O5eMFMVpbm0Bwe4IV9TVSHwsdK34OKZqTS0j1wQjWKCAz9M5lfkMmykmy+evNKcjKCozlMYxKPGwZicWJxwx92NlAVClOYlcqy2dl090dp74kc+76j8TirynKHPUErDzAG9j4Nv/9n6Kw7cdncS2DpjZCWAyv/DpL1V9bZTHqin0gTkehjccNlX/8TSSI89+krSAueWBI40trDW775AtEz1AMDpCYnMSMtSNwYWrsHuGZZMTPSgkTjcYKBJARIEuHXW+tIDwaYm5/Bp66uwBhb3bKqLHdc+zEez+xq4MXKEHkZQTJSkqlqsr8WirJTef1gK28cPl61taosh6WzsvnEukXsbwrznWcr6Y3EGIjGuXB+PqvKcukZiHKwuZvdRzvZVtvBzWtKuefdq93aPeVnLdWw83F4/iv2dTADKq6B1Cxo3AXzLoWr/hWC2o3JaEy7RA/wP6/W8K9P7qQkJ42v3ryS5bOzyckI8ttt9fzTY9sAuGJxIe84r5SrlxWzp76T321voLGzjysWF9LSPcD+pi5auwdo74lw+5vmcfOaxOkN70hrD72RGIsKs0Zd2h2Ixie1/l8lqHAItj0Mz/0bxO2vS5bcAH/3ACRrt87jNS0TvTGGn79awzee2UdXX/SU5d97z3n8zarhq3Wmjd426G62dZ/GQHYpBLVJ6rQXj0G0zzZjTDrDCT0eH355pA/aDkEgaD+roAIiPXDPCuhttaX3678OGQWQln3q+9WYTEmrG68RET5wSTnnzc3jnmcr6R6I0hweYH5BJn9/2XwuWjDT7RBHLx6zj9HUVw7+TI70QGYhBFJscq/8PdRvO3X9eZfarl5LVmnfIX4Rj0H9VuhqtO3Muxog1g/nfQDSnQvQHbXQsMN2F5BZcOJ7m/bYKpOSc+1FmF++xyZqxK5bshrSc2H2eXa0pq6jdqi+qmdh1a22qWPdZvvegR44/Ir9exs0+zwY6LZJ/vpvwIUfBr1r3TUJVaJPGL3tsOMx2PNbOPgSYGDmIkjPt/90y26C7NmQlGwTdzwKbQdtMu+ote8xsVM/t3ApLFoHNS/Dgisgrxzqt9sOomJOf/4r3mkT/+r3nFhf2lkPLfttEslfaE8ePc22rnWgG5orbfvn4mU27nCjHQQiowCyim2y6Q5B9ll+URkDoX3Q3WR/beSVT7+WF7EINGy3x7F5P3Qetcc4NdteqAymQ/VzTmI+SWoOzLsE+sNQ81fAQFLQnvCTU+z3G26CaO+J7wukwoIr7Q1J9dugucqeOHpajq8TzICULHtswD5Pz7cJvOJq23omFrHHee8G+0vxsk/BOX8zOd+Tmp5VN54Wix5PWKcr2dS9AaG98Odv26SaMgPOvdUm9KZdNpHWb7M/sU82+M+cnguLr4PzP2gTbf1W+znZJZA6Y/jtRvpsHyJ//S/Y/gjEI/bz5r8ZktPsssadJ71JsA2rTkMCw59sFq6DSC+0VkNmkY0pHrEJorPOJomhcufCnIvse7rq7S+O0D77PeUvsO8vuwDWvM+eGLxaamw9ANV/solWxJ4oJWBPnANOUm4/Yk/Y7YeP12dnFdvk3nbIrp+WbftiN3G4+GNQvAKKltrvYaAb/vxNW1JH7En33Nvg8Ks2Yfe1Q18HFK+EvHmQO8/GFe2FpTdBwaITY45F7d9hT4uNu+Rcu532w3ZbBYshkFCVAr6jiX6y9XXYEldHrX101tkSVyDFJuZAEDJm2jrxxl1Q9UeQJMidA8tvtv+gzZW2tDYQtj/BD79sPzslC255EOZddmr9eXcLtFRBxxH7U3nuJXZb6fkTU/cZ6bWxbvqpLbkZbJ1sxTW2pN8dsj/Z03LsNnvb7E/93jb7PaTn2aZxxcvt/sUi9mRT/Tw077MlzbQcm5z6OqCv0yaLpKD9lZJZaE9KufPs5+39na1mSM2y8YX2QVYRzL3Yfhc1fzkx/nPeZkusgaAt+UZ67edWXGtPGgUV0HoQ9v+v/UUSGKwSE0jJsNUaGHusJMnOT8u2J7wFb7GxHt1qT7bhJptwMwttn+jBdJsIWw/av43WA9C02ybJ3tazf/ezVtmTV/58m1TnXGRjhxPrxgf/Z716UlNTRhP92fS02hJW/VZb5dGw43jpKiXLJqO0XJuYTdyWYBp32fpvnJ+/MduenkCqLT33d9mSWDxmO2CKR50EVmL/cXPmwP5nbSlpUDADZsyyz1e805bAMgv1gtXpRPtPbK1hjC2x7n0aKp+x33l6HqRk2hNh3ebTf1Z6vv28SLc9OcejdipJ9pgPPsYiPc+erIqX23jzym0Sz55tTy7BDBt7T4s9mc9cZGNWahSmX6Lf9giE9kBbjU3ea95n/2kbdtrSVNkF9pFVZNd55L3H35uaDUXL7D96MN0m6t42+1O3y7nXK5hh/2ELl9p/1tQZsOoWm7yzik+9WBodsAkkLffUkldfp42xcClkFY5vv9XZdbfY0ntLlS1t55QeLy0bY6sicsrstZH0vFNblXQ12msO2x+xr+dcZH+RBDNsVVnt6/ZzBsL2bycpGZZcP/2uLagpN70SvTHwn/NtcgZb9zt4wQhsiTs2wCl1ytd8BS74+zM3L4z221L5mZqcKaWUC6ZH88rWg/Di121Tr942WHmLrWL52+/YUnt7jV1WstpWqxx9w5bqsopth0kFIxjzU2/qUEr5nL8T/UDY3nU36Kp/sXWhYKto5lxwfFlatm0+tuDKKQtPKaW8wN+JftZK+PDzsPUhWyc6mOSVUkod4+9ED1B6nn0opZQall5hVEqpBKeJXimlEpwmeqWUSnCa6JVSKsFNWqIXketEZJ+IVInI5yZrO0oppc5sUhK9iASA7wHXA8uA20Rk2WRsSyml1JlNVon+QqDKGHPAGDMA/BK4aZK2pZRS6gwmK9GXAkeGvK515imllJpirt0wJSLrgfXOy7CI7BvHxxUAzeOPyjd0fxOb7m9im8j9nTeSlSYr0dcBc4a8LnPmHWOMuQ+4byI2JiKbRtKDW6LQ/U1sur+JzY39nayqm9eBChGZLyIpwK3AU5O0LaWUUmcwKSV6Y0xURD4OPAMEgPuNMbsmY1tKKaXObNLq6I0xG4ANk/X5J5mQKiAf0f1NbLq/iW3K99cTI0wppZSaPNoFglJKJThfJ/pE7GZBROaIyPMisltEdonI3c78fBF5VkT2O9M8Z76IyHed72C7iPiyc34RCYjIFhF52nk9X0Q2Ovv1iHNRHxFJdV5XOcvL3Yx7LEQkV0R+JSJ7RWSPiFySyMdXRD7l/C3vFJGHRSQt0Y6viNwvIk0isnPIvFEfUxG53Vl/v4jcPlHx+TbRJ3A3C1Hg08aYZcDFwF3Ofn0OeM4YUwE857wGu/8VzmM9cO/Uhzwh7gb2DHn9deAeY8wioA2405l/J9DmzL/HWc9v/gv4gzHmHOBc7H4n5PEVkVLgH4C1xpgV2MYZt5J4x/cB4LqT5o3qmIpIPvAl4CJs7wJfGjw5jJsxxpcP4BLgmSGvPw983u24JmE/fwNcDewDSpx5JcA+5/kPgduGrH9sPb88sPdZPAdcBTwNCPaGkuSTjzW2JdclzvNkZz1xex9Gsa85wMGTY07U48vxu+TzneP1NHBtIh5foBzYOdZjCtwG/HDI/BPWG8/DtyV6pkE3C87P1jXARqDYGFPvLGoAip3nifA9fAf4LBB3Xs8E2o0xUef10H06tr/O8g5nfb+YD4SAnzpVVT8WkUwS9PgaY+qAbwKHgXrs8dpM4h7foUZ7TCftWPs50Sc0EckCHgc+aYzpHLrM2NN9QjSXEpG3AU3GmM1uxzJFkoHzgHuNMWuAbo7/pAcS7vjmYTs0nA/MBjI5tYoj4bl9TP2c6M/azYJfiUgQm+QfMsY84cxuFJESZ3kJ0OTM9/v3cClwo4gcwvZyehW2DjtXRAbv8xi6T8f211meA7RMZcDjVAvUGmM2Oq9/hU38iXp83wocNMaEjDER4AnsMU/U4zvUaI/ppB1rPyf6hOxmQUQE+Amwxxjz7SGLngIGr8Lfjq27H5z/AedK/sVAx5Cfi55njPm8MabMGFOOPYZ/Msa8F3geeJez2sn7O/g9vMtZ3zelX2NMA3BERJY4s9YBu0nQ44utsrlYRDKcv+3B/U3I43uS0R7TZ4BrRCTP+SV0jTNv/Ny+gDHOix83AJVANfAvbsczQft0GfYn3nZgq/O4AVtP+RywH/gjkO+sL9jWR9XADmzrBtf3Y4z7fiXwtPN8AfAaUAU8BqQ689Oc11XO8gVuxz2G/VwNbHKO8ZNAXiIfX+DfgL3ATuDnQGqiHV/gYew1iAj2V9udYzmmwIecfa8C7pio+PTOWKWUSnB+rrpRSik1AprolVIqwWmiV0qpBKeJXimlEpwmeqWUSnCa6JVSKsFpoldKqQSniV4ppRLc/wcn8y771PiqUQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(average_score_replay_buffer)\n",
    "plt.plot(average_score_replay_per_buffer)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
